#include "../pch.h"
#include "REFEncoding.h"
#include "REFUTF16Encoding.h"

namespace BFX
{

REFUTF16Encoding::REFUTF16Encoding(bool bBigEndian)
		: REFEncoding(bBigEndian ? 1201 : 1200)	//Set the data item
	{
		m_bBigEndian = bBigEndian;
	}

REFUTF16Encoding::~REFUTF16Encoding()
{
}

int REFUTF16Encoding::GetMaxByteCount(int nCharCount) const
{
	BFX_REQUIRE (nCharCount >= 0 && "Non-negative number required.");

	// Characters would be # of characters + 1 in case left over high surrogate is ? * max fallback
	long long lByteCount = (long long)nCharCount + 1;

	// 2 bytes per char
	lByteCount <<= 1;

	BFX_ENSURE1(lByteCount <= 0x7fffffff, "Too many characters. The resulting number of bytes is larger than what can be returned as an int.");

	return (int)lByteCount;
}

int REFUTF16Encoding::GetMaxCharCount(int nByteCount) const
{
	BFX_REQUIRE1 (nByteCount >= 0, "Non-negative number required.");

	// long because nByteCount could be biggest int.
	// 1 char per 2 bytes.  Round up in case 1 left over in decoder.
	// Round up using &1 in case nByteCount is max size
	// Might also need an extra 1 if there's a left over high surrogate in the decoder.
	long long nCharCount = (long long)(nByteCount >> 1) + (nByteCount & 1) + 1;

	BFX_ENSURE1(nCharCount <= 0x7fffffff, "Too many characters. The resulting number of bytes is larger than what can be returned as an int.");

	return (int)nCharCount;
}
int REFUTF16Encoding::GetByteCount(const UCHAR* pChars, int nCount) const
{
	// Validate the parameters.
	BFX_REQUIRE1 (pChars != NULL, "Null pointer disallowed.");
	BFX_REQUIRE1 (nCount >= 0 , "Non-negative number required.");

	BFX_ASSERT(pChars != NULL);
	BFX_ASSERT(nCount >= 0);

	// Starts by assuming each char gets 2 bytes
	int nByteCount = nCount * 2;
	BFX_ENSURE1 (nByteCount >= 0, "Too many characters. The resulting number of bytes is larger than what can be returned as an int.");

	const UCHAR* pCharStart = pChars;
	const UCHAR* pCharEnd = pChars + nCount;
	UCHAR  charLeftOver = (UCHAR)0;

	bool bWasHereBefore = false;

	// Need -1 to check 2 at a time.  If we have an even #, longChars will go
	// from pLongEnd - 1/2 long to pLongEnd + 1/2 long.  If we're odd, longChars
	// will go from pLongEnd - 1 long to pLongEnd. (Might not get to use this)
	UINT64* pLongEnd = (UINT64*)(pCharEnd - 3);

	bool bFallingBack = false;

	UCHAR ch;
TryAgain:

	while (((ch = bFallingBack ? '?' : 0) != 0) || pChars < pCharEnd)
	{
		bFallingBack = false;

		// First unwind any fallback
		if (ch == 0)
		{
			// No fallback, maybe we can do it fast
#if BIGENDIAN       // If endianess is backwards then each pair of bytes would be backwards.
			if ( m_bBigEndian &&
#else
			if ( !m_bBigEndian &&
#endif // BIGENDIAN

				charLeftOver == 0 && (((size_t)pChars) & 3) == 0)
			{
				// Need new const UCHAR* so we can check 4 at a time
				UINT64* longChars = (UINT64*)pChars;

				while (longChars < pLongEnd)
				{
					if (((UINT64)(0x8000800080008000LL) & *longChars) != 0)
					{
						// See if all 4 of those are bad...
						// Surrogate pairs are "backwards" for little endian machines read in as one
						// Mask off surrogate bits
						UINT64 uTemp = (0xd800d800d800d800LL & *longChars) ^ 0xd800d800d800d800LL;

						// Check each of the 4 chars.  0 for those 16 bits means it was a surrogate
						if ((uTemp & 0xFFFF000000000000LL) == 0 ||
							(uTemp & 0x0000FFFF00000000LL) == 0 ||
							(uTemp & 0x00000000FFFF0000LL) == 0 ||
							(uTemp & 0x000000000000FFFFLL) == 0)
						{
							// Does it happen to have 4 good surrogates?
#if BIGENDIAN
							if (((0xdc00dc00dc00dc00LL & *longChars) ^ 0xd800dc00d800dc00LL) != 0)
#else
							if (((0xdc00dc00dc00dc00LL & *longChars) ^ 0xdc00d800dc00d800LL) != 0)
#endif
							{
								// If it wasn't 4 good surrogates then fall through to slow loop
								break;
							}
						}

						// It was a good value, we'll use these 4 characters
					}

					// We already counted these four chars, go to next long.
					longChars++;
				}

				pChars = (const UCHAR*)longChars;

				if (pChars >= pCharEnd)
					break;
			}

			// No fallback, just get next char
			ch = *pChars;
			pChars++;
		}
		else
		{
			// We weren't preallocating fallback space.
			nByteCount+=2;
		}

		// Check for high or low surrogates
		if (ch >= 0xd800 && ch <= 0xdfff)
		{
			// Was it a high surrogate?
			if (ch <= 0xdbff)
			{
				// Its a high surrogate, if we already had a high surrogate do its fallback
				if (charLeftOver > 0)
				{
					// Unwind the current character, this should be safe because we
					// don't have leftover data in the fallback, so chars must have
					// advanced already.
					BFX_ASSERT(pChars > pCharStart);	// Expected pChars to have advanced in unexpected high surrogate
					pChars--;

					// If previous high surrogate deallocate 2 bytes
					nByteCount -= 2;

					// Fallback the previous surrogate
					bFallingBack = true;
					pChars ++;

					// Now no high surrogate left over
					charLeftOver = (UCHAR)0;
					continue;
				}

				// Remember this high surrogate
				charLeftOver = ch;
				continue;
			}


			// Its a low surrogate
			if (charLeftOver == 0)
			{
				// Expected a previous high surrogate.
				// Don't count this one (we'll count its fallback if necessary)
				nByteCount -= 2;

				// fallback this one
				bFallingBack = true;
				pChars ++;

				continue;
			}

			// Valid surrogate pair, add our charLeftOver
			charLeftOver = (UCHAR)0;
			continue;
		}
		else if (charLeftOver > 0)
		{
			// Expected a low surrogate, but this char is normal

			// Rewind the current character, fallback previous character.
			// this should be safe because we don't have leftover data in the
			// fallback, so chars must have advanced already.
			BFX_ASSERT1(pChars > pCharStart, "Expected chars to have advanced when expected low surrogate");
			pChars--;

			// fallback previous chars
			bFallingBack = true;
			pChars ++;

			// Ignore charLeftOver or throw
			nByteCount-=2;
			charLeftOver = (UCHAR)0;

			continue;
		}

		// Ok we had something to add (already counted)
	}

	// Don't allocate space for left over char
	if (charLeftOver > 0)
	{
		nByteCount -= 2;

		// If we have to flush, stick it in fallback and try again
		// Throw it, using our complete character
		BFX_ENSURE1 (!bWasHereBefore, "Recursive fallback not allowed for character.");

		// Need to initialize fallback buffer?
		bFallingBack = true;
		pChars ++;

		charLeftOver = (UCHAR)0;
		bWasHereBefore = true;
		goto TryAgain;
	}

	return nByteCount;
}

int REFUTF16Encoding::GetBytes(const UCHAR* pChars, int nCharCount, OUT BYTE* pBytes, int nByteCount) const
{
	// Validate the parameters.
	BFX_ASSERT1 (pChars != NULL && pBytes != NULL, "Null reference of argument disallowed.");
	BFX_ASSERT1 (nCharCount >= 0 && nByteCount >= 0, "Non-negative number required.");

	BFX_ASSERT(pChars!=NULL);
	BFX_ASSERT(nByteCount >=0);
	BFX_ASSERT(nCharCount >=0);
	BFX_ASSERT(pBytes!=NULL);

	UCHAR charLeftOver = (UCHAR)0;
	UCHAR ch;
	bool bWasHereBefore = false;


	BYTE* pByteEnd = pBytes + nByteCount;
	const UCHAR* pCharEnd = pChars + nCharCount;
	BYTE* pByteStart = pBytes;
	const UCHAR* pCharStart = pChars;

	bool bFallingBack = false;

TryAgain:
	while (((ch = (!bFallingBack) ? (UCHAR)0 : '?') != 0) || pChars < pCharEnd)
	{
		bFallingBack = false;
		// First unwind any fallback
		if (ch == 0)
		{
			// No fallback, maybe we can do it fast
#if BIGENDIAN           // If endianess is backwards then each pair of bytes would be backwards.
			if ( m_bBigEndian &&
#else
			if ( !m_bBigEndian &&
#endif // BIGENDIAN
				(((size_t)pChars) & 3) == 0 && (((size_t)pBytes) & 3) == 0 &&
				charLeftOver == 0)
			{
				// Need -1 to check 2 at a time.  If we have an even #, longChars will go
				// from pLongEnd - 1/2 long to pLongEnd + 1/2 long.  If we're odd, longChars
				// will go from pLongEnd - 1 long to pLongEnd. (Might not get to use this)
				// We can only go iCount units (limited by shorter of char or byte buffers.
				UINT64* pLongEnd = (UINT64*)(pChars - 3 +
					(((pByteEnd - pBytes) >> 1 < pCharEnd - pChars) ?
					(pByteEnd - pBytes) >> 1 : pCharEnd - pChars));

				// Need new UCHAR* so we can check 4 at a time
				UINT64* longChars = (UINT64*)pChars;
				UINT64* longBytes = (UINT64*)pBytes;

				while (longChars < pLongEnd)
				{
					if (((UINT64)(0x8000800080008000LL) & *longChars) != 0)
					{
						// See if all 4 of those are bad...
						// Surrogate pairs are "backwards" for little endian machines read in as one
						// Mask off surrogate bits
						UINT64 uTemp = (0xd800d800d800d800LL & *longChars) ^ 0xd800d800d800d800LL;

						// Check each of the 4 chars.  0 for those 16 bits means it was a surrogate
						if ((uTemp & 0xFFFF000000000000LL) == 0 ||
							(uTemp & 0x0000FFFF00000000LL) == 0 ||
							(uTemp & 0x00000000FFFF0000LL) == 0 ||
							(uTemp & 0x000000000000FFFFLL) == 0)
						{
							// Does it happen to have 4 good surrogates?
#if BIGENDIAN
							if (((0xdc00dc00dc00dc00LL & *longChars) ^ 0xd800dc00d800dc00LL) != 0)
#else
							if (((0xdc00dc00dc00dc00LL & *longChars) ^ 0xdc00d800dc00d800LL) != 0)
#endif // BIGENDIAN
							{
								// If it wasn't 4 good surrogates then fall through to slow loop
								break;
							}

							// Use these 4 lucky ordered surrogate pairs.
						}

						// Use these 4 high, but not surrogates
					}

					// We can use these 4 chars.
					*longBytes = *longChars;
					longChars++;
					longBytes++;
				}

				pChars = (UCHAR*)longChars;
				pBytes = (BYTE*)longBytes;

				if (pChars >= pCharEnd)
					break;
			}
			// Not aligned, but maybe we can still be somewhat faster
			// Also somehow this optimizes the above loop?  It seems to cause something above
			// to get enregistered, but I haven't figured out how to make that happen without this loop.
			else if ((charLeftOver == 0) &&
#if BIGENDIAN
				m_bBigEndian &&
#else
				!m_bBigEndian &&
#endif // BIGENDIAN

				(((size_t)pChars) & 3) != (((size_t)pBytes) & 3) &&  // Only do this if chars & bytes are out of line, otherwise faster loop'll be faster next time
				(((size_t)(pBytes)) & 1) == 0 )
			{
				// # to use
				long iCount = ((pByteEnd - pBytes) >> 1 < pCharEnd - pChars) ?
					(pByteEnd - pBytes) >> 1 : pCharEnd - pChars;

				// Need new UCHAR*
				UCHAR* charOut = ((UCHAR*)pBytes);     // a UCHAR* for our output
				const UCHAR* tempEnd = pChars + iCount - 1; // Our end pointer

				while (pChars < tempEnd)
				{
					if (*pChars >= (UCHAR)0xd800 && *pChars <= (UCHAR)0xdfff)
					{
						// break for fallback for low surrogate
						if (*pChars >= 0xdc00)
							break;

						// break if next one's not a low surrogate (will do fallback)
						if (*(pChars+1) < 0xdc00 || *(pChars+1) > 0xdfff)
							break;

						// They both exist, use them
					}
					// If 2nd char is surrogate & this one isn't then only add one
					else if (*(pChars+1) >= (UCHAR)0xd800 && *(pChars+1) <= 0xdfff)
					{
						*charOut = *pChars;
						charOut++;
						pChars++;
						continue;
					}

					*charOut = *pChars;
					*(charOut+1) = *(pChars+1);
					charOut+=2;
					pChars+=2;

				}

				pBytes=(BYTE*)charOut;

				if (pChars >= pCharEnd)
					break;
			}

			// No fallback, just get next char
			ch = *pChars;
			pChars++;
		}

		// Check for high or low surrogates
		if (ch >= 0xd800 && ch <= 0xdfff)
		{
			// Was it a high surrogate?
			if (ch <= 0xdbff)
			{
				// Its a high surrogate, see if we already had a high surrogate
				if (charLeftOver > 0)
				{
					// Unwind the current character, this should be safe because we
					// don't have leftover data in the fallback, so chars must have
					// advanced already.
					BFX_ASSERT1(pChars > pCharStart, "Expected chars to have advanced in unexpected high surrogate");
					pChars--;

					// Fallback the previous surrogate
					bFallingBack = true;
					pChars ++;

					charLeftOver = (UCHAR)0;
					continue;
				}

				// Remember this high surrogate
				charLeftOver = ch;
				continue;
			}

			// Its a low surrogate
			if (charLeftOver == 0)
			{
				// We'll fall back this one
				bFallingBack = true;
				pChars ++;

				continue;
			}

			// Valid surrogate pair, add our charLeftOver
			if (pBytes + 3 >= pByteEnd)
			{
				BFX_ASSERT(!bFallingBack);
				// Not enough room to add this surrogate pair

				// If we don't have enough room, then either we should've advanced a while
				// or we should have pBytes==pByteStart and throw below
				BFX_ASSERT1((pChars > pCharStart + 1 || pBytes == pByteStart), "Expected chars to have when no room to add surrogate pair");
				pChars-=2;                                      // Didn't use either surrogate

				// Throw maybe (if no bytes written)
				BFX_ENSURE1(pBytes != pByteStart, "The output byte buffer is too small to contain the encoded data");

				charLeftOver = (UCHAR)0;                          // we'll retry it later
				break;                                              // Didn't throw, but stop 'til next time.
			}

			if (m_bBigEndian)
			{
				*(pBytes++) = (BYTE)(charLeftOver >> 8);
				*(pBytes++) = (BYTE)charLeftOver;
			}
			else
			{
				*(pBytes++) = (BYTE)charLeftOver;
				*(pBytes++) = (BYTE)(charLeftOver >> 8);
			}

			charLeftOver = (UCHAR)0;
		}
		else if (charLeftOver > 0)
		{
			// Expected a low surrogate, but this char is normal

			// Rewind the current character, fallback previous character.
			// this should be safe because we don't have leftover data in the
			// fallback, so chars must have advanced already.
			BFX_ASSERT1(pChars > pCharStart, "Expected chars to have advanced after expecting low surrogate");
			pChars--;

			// fallback previous chars
			bFallingBack = true;
			pChars ++;

			// Ignore charLeftOver or throw
			charLeftOver = (UCHAR)0;
			continue;
		}

		// Ok, we have a char to add
		if (pBytes + 1 >= pByteEnd)
		{
			BFX_ASSERT(!bFallingBack);

			// Couldn't add this char

			// Lonely charLeftOver (from previous call) would've been caught up above,
			// so this must be a case where we've already read an input char.
			BFX_ASSERT(pChars > pCharStart);
			pChars--;                                       // Not using this char

			// Throw maybe (if no bytes written)
			BFX_ENSURE(pBytes != pByteStart);
			break;                                              // just stop
		}

		if (m_bBigEndian)
		{
			*(pBytes++) = (BYTE)(ch >> 8);
			*(pBytes++) = (BYTE)ch;
		}
		else
		{
			*(pBytes++) = (BYTE)ch;
			*(pBytes++) = (BYTE)(ch >> 8);
		}
	}

	// Don't allocate space for left over char
	if (charLeftOver > 0)
	{
		// If we aren't flushing we need to fall this back
		BFX_ENSURE1(!bWasHereBefore, "Recursive fallback not allowed for character.");

		// If we have to flush, stick it in fallback and try again
		// If we're not flushing, this'll remember the left over character.
		bFallingBack = true;
		pChars++;

		charLeftOver = (UCHAR)0;
		bWasHereBefore = true;
		goto TryAgain;
	}

	return (int)(pBytes - pByteStart);
}

int REFUTF16Encoding::GetCharCount(const BYTE* pBytes, int nCount, REFDecoder* pDecoder) const
{
	// Validate the parameters.
	BFX_REQUIRE (pBytes != NULL && "Null pointer disallowed");
	BFX_REQUIRE (nCount >= 0 && "Non-negative number required.");

	UTF16Decoder* pUtf16Decoder = (UTF16Decoder*)pDecoder;

	const BYTE* pByteEnd = pBytes + nCount;
	const BYTE* pByteStart = pBytes;

	// Need last vars
	int lastByte = -1;
	UCHAR lastChar = (UCHAR)0;

	// Start by assuming same # of chars as bytes
	int nCharCount = nCount >> 1;

	// Need -1 to check 2 at a time.  If we have an even #, longBytes will go
	// from pLongEnd - 1/2 long to pLongEnd + 1/2 long.  If we're odd, longBytes
	// will go from pLongEnd - 1 long to pLongEnd. (Might not get to use this)
	UINT64* pLongEnd = (UINT64*)(pByteEnd - 7);

	// For fallback we may need a fallback buffer
	REF<REFDecoderFallbackBuffer> pFallbackBuffer;

	if (pUtf16Decoder != NULL)
	{
		lastByte = pUtf16Decoder->m_lastByte;
		lastChar = pUtf16Decoder->m_lastChar;

		// Assume extra char if last char was around
		if (lastChar > 0)
			nCharCount++;

		// Assume extra char if extra last byte makes up odd # of input bytes
		if (lastByte >= 0 && (nCount & 1) == 1)
		{
			nCharCount++;
		}

		// Shouldn't have anything in fallback buffer for GetCharCount
		// (don't have to check m_throwOnOverflow for count)
		BFX_ASSERT1(!pDecoder->HasFallbackBuffer() || pDecoder->GetFallbackBuffer()->GetRemaining() == 0, "Expected empty fallback buffer at start");
	}

	while (pBytes < pByteEnd)
	{
		// If we're aligned then maybe we can do it fast
		// This'll hurt if we're unaligned because we'll always test but never be aligned
#if BIGENDIAN
		if (m_bBigEndian &&
#else // BIGENDIAN
		if (!m_bBigEndian &&
#endif // BIGENDIAN
			(((size_t)pBytes) & 3) == 0 &&
			lastByte == -1 && lastChar == 0)
		{
			// Need new UCHAR* so we can check 4 at a time
			UINT64* longBytes = (UINT64*)pBytes;

			while (longBytes < pLongEnd)
			{
				if (((UINT64)(0x8000800080008000LL) & *longBytes) != 0)
				{
					// See if all 4 of those are bad...
					// Surrogate pairs are "backwards" for little endian machines read in as one
					// Mask off surrogate bits
					UINT64 uTemp = (0xd800d800d800d800LL & *longBytes) ^ 0xd800d800d800d800LL;

					// Check each of the 4 chars.  0 for those 16 bits means it was a surrogate
					if ((uTemp & 0xFFFF000000000000LL) == 0 ||
						(uTemp & 0x0000FFFF00000000LL) == 0 ||
						(uTemp & 0x00000000FFFF0000LL) == 0 ||
						(uTemp & 0x000000000000FFFFLL) == 0)
					{
						// Does it happen to have 4 good surrogates?
#if BIGENDIAN
						if (((0xdc00dc00dc00dc00LL & *longBytes) ^ 0xd800dc00d800dc00LL) != 0)
#else
						if (((0xdc00dc00dc00dc00LL & *longBytes) ^ 0xdc00d800dc00d800LL) != 0)
#endif // BIGENDIAN
						{
							// If it wasn't 4 good surrogates then fall through to slow loop
							break;
						}

						// Use these 4 lucky ordered surrogate pairs.
					}

					// Use these 4 high, but not surrogates
				}

				// We can use these 4 chars.
				longBytes++;
			}

			pBytes = (BYTE*)longBytes;

			if (pBytes >= pByteEnd)
				break;
		}

		// Get 1st byte
		if (lastByte < 0)
		{
			lastByte = *pBytes++;
			if (pBytes >= pByteEnd) break;
		}

		// Get full char
		UCHAR ch;
		if (m_bBigEndian)
		{
			ch = (UCHAR)(lastByte << 8 | *(pBytes++));
		}
		else
		{
			ch = (UCHAR)(*(pBytes++) << 8 | lastByte);
		}
		lastByte = -1;

		// See if the char's valid
		if (ch >= 0xd800 && ch <= 0xdfff)
		{
			// Was it a high surrogate?
			if (ch <= 0xdbff)
			{
				// Its a high surrogate, if we had one then do fallback for previous one
				if (lastChar > 0)
				{
					// Ignore previous bad high surrogate
					nCharCount--;

					// Get fallback for previous high surrogate
					// Note we have to reconstruct bytes because some may have been in decoder
					SequenceT<BYTE> bytesUnknown;
					BYTE* pBuf = bytesUnknown.GetBuffer(2);

					if (m_bBigEndian)
					{
						pBuf[0] = (BYTE)(lastChar >> 8);
						pBuf[1] = (BYTE)lastChar;
					}
					else
					{
						pBuf[0] = (BYTE)lastChar;
						pBuf[1] = (BYTE)(lastChar >> 8);
					}
					bytesUnknown.ReleaseBuffer(2);

					if (pFallbackBuffer == NULL)
					{
						if (pDecoder == NULL)
							pFallbackBuffer = GetDecoderFallback()->CreateFallbackBuffer();
						else
							pFallbackBuffer = pDecoder->GetFallbackBuffer();

						// Set our internal fallback interesting things.
						pFallbackBuffer->Initialize(pByteStart, NULL);
					}

					// Get fallback.
					nCharCount += pFallbackBuffer->Fallback(bytesUnknown, pBytes);
				}

				// Ignore the last one which fell back already,
				// and remember the new high surrogate
				lastChar = ch;
				continue;
			}

			// Its a low surrogate
			if (lastChar == 0)
			{
				// Expected a previous high surrogate
				nCharCount--;

				// Get fallback for this low surrogate
				// Note we have to reconstruct bytes because some may have been in decoder
				SequenceT<BYTE> bytesUnknown;
				if (m_bBigEndian)
				{
					bytesUnknown.Append((BYTE)(ch >> 8));
					bytesUnknown.Append((BYTE)ch);
				}
				else
				{
					bytesUnknown.Append((BYTE)ch);
					bytesUnknown.Append((BYTE)(ch >> 8));
				}

				if (pFallbackBuffer == NULL)
				{
					if (pDecoder == NULL)
						pFallbackBuffer = GetDecoderFallback()->CreateFallbackBuffer();
					else
						pFallbackBuffer = pDecoder->GetFallbackBuffer();

					// Set our internal fallback interesting things.
					pFallbackBuffer->Initialize(pByteStart, NULL);
				}

				nCharCount += pFallbackBuffer->Fallback(bytesUnknown, pBytes);

				// Ignore this one (we already did its fallback)
				continue;
			}

			// Valid surrogate pair, already counted.
			lastChar = (UCHAR)0;
		}
		else if (lastChar > 0)
		{
			// Had a high surrogate, expected a low surrogate
			// Uncount the last high surrogate
			nCharCount--;

			// fall back the high surrogate.
			SequenceT<BYTE> bytesUnknown;
			if (m_bBigEndian)
			{
				bytesUnknown.Append((BYTE)(lastChar >> 8));
				bytesUnknown.Append((BYTE)lastChar);
			}
			else
			{
				bytesUnknown.Append((BYTE)lastChar);
				bytesUnknown.Append((BYTE)(lastChar >> 8));
			}

			if (pFallbackBuffer == NULL)
			{
				if (pDecoder == NULL)
					pFallbackBuffer = GetDecoderFallback()->CreateFallbackBuffer();
				else
					pFallbackBuffer = pDecoder->GetFallbackBuffer();

				// Set our internal fallback interesting things.
				pFallbackBuffer->Initialize(pByteStart, NULL);
			}

			// Already subtracted high surrogate
			nCharCount += pFallbackBuffer->Fallback(bytesUnknown, pBytes);

			// Not left over now, clear previous high surrogate and continue to add current char
			lastChar = (char)0;
		}

		// Valid char, already counted
	}

	// Extra space if we can't use decoder
	if (pDecoder == NULL || pDecoder->IsFlushRequired())
	{
		if (lastChar > 0)
		{
			// No hanging high surrogates allowed, do fallback and remove count for it
			nCharCount--;
			SequenceT<BYTE> bytesUnknown;
			if (m_bBigEndian)
			{
				bytesUnknown.Append((BYTE)(lastChar >> 8));
				bytesUnknown.Append((BYTE)lastChar);
			}
			else
			{
				bytesUnknown.Append((BYTE)lastChar);
				bytesUnknown.Append((BYTE)(lastChar >> 8));
			}

			if (pFallbackBuffer == NULL)
			{
				if (pDecoder == NULL)
					pFallbackBuffer = GetDecoderFallback()->CreateFallbackBuffer();
				else
					pFallbackBuffer = pDecoder->GetFallbackBuffer();

				// Set our internal fallback interesting things.
				pFallbackBuffer->Initialize(pByteStart, NULL);
			}

			nCharCount += pFallbackBuffer->Fallback(bytesUnknown, pBytes);

			lastChar = (char)0;
		}

		if (lastByte >= 0)
		{
			if (pFallbackBuffer == NULL)
			{
				if (pDecoder == NULL)
					pFallbackBuffer = GetDecoderFallback()->CreateFallbackBuffer();
				else
					pFallbackBuffer = pDecoder->GetFallbackBuffer();

				// Set our internal fallback interesting things.
				pFallbackBuffer->Initialize(pByteStart, NULL);
			}

			// No hanging odd bytes allowed if must flush
			SequenceT<BYTE> bytesUnknown;
			bytesUnknown.Append((BYTE)lastByte);
			nCharCount += pFallbackBuffer->Fallback(bytesUnknown, pBytes);
			lastByte = -1;
		}
	}

	// If we had a high surrogate left over, we can't count it
	if (lastChar > 0)
		nCharCount--;

	// Shouldn't have anything in fallback buffer for GetCharCount
	// (don't have to check m_throwOnOverflow for count)
	BFX_ASSERT1(pFallbackBuffer == NULL || pFallbackBuffer->GetRemaining() == 0, "Expected empty fallback buffer at end");

	return nCharCount;
}

int REFUTF16Encoding::GetChars(const BYTE* pBytes, int nByteCount, OUT UCHAR* pChars, int nCharCount, REFDecoder* pDecoder) const
{
	BFX_ASSERT(pChars!=NULL);
	BFX_ASSERT(nByteCount >=0);
	BFX_ASSERT(nCharCount >=0);
	BFX_ASSERT(pBytes!=NULL);

	UTF16Decoder* pUtf16Decoder = (UTF16Decoder*) pDecoder;

	// Need last vars
	int lastByte = -1;
	UCHAR lastChar = (UCHAR)0;

	// Get our decoder (but don't clear it yet)
	if (pUtf16Decoder != NULL)
	{
		lastByte = pUtf16Decoder->m_lastByte;
		lastChar = pUtf16Decoder->m_lastChar;

		// Shouldn't have anything in fallback buffer for GetChars
		// (don't have to check m_throwOnOverflow for chars)
		BFX_ASSERT1(!pUtf16Decoder->HasFallbackBuffer() || pUtf16Decoder->GetFallbackBuffer()->GetRemaining() == 0,
			"Expected empty fallback buffer at start");
	}

	// For fallback we may need a fallback buffer
	REF<REFDecoderFallbackBuffer> pFallbackBuffer;

	const BYTE* pByteEnd = pBytes + nByteCount;
	UCHAR* pCharEnd = pChars + nCharCount;
	const BYTE* pByteStart = pBytes;
	UCHAR* pCharStart = pChars;

	while (pBytes < pByteEnd)
	{
		// If we're aligned then maybe we can do it fast
		// This'll hurt if we're unaligned because we'll always test but never be aligned
#if BIGENDIAN
		if (m_bBigEndian &&
#else // BIGENDIAN
		if (!m_bBigEndian &&
#endif // BIGENDIAN
			(((size_t)pChars) & 3) == 0 && (((size_t)pBytes) & 3) == 0 &&
			lastByte == -1 && lastChar == 0)
		{
			// Need -1 to check 2 at a time.  If we have an even #, longChars will go
			// from pLongEnd - 1/2 long to pLongEnd + 1/2 long.  If we're odd, longChars
			// will go from pLongEnd - 1 long to pLongEnd. (Might not get to use this)
			// We can only go iCount units (limited by shorter of char or byte buffers.
			UINT64* pLongEnd = (UINT64*)(pBytes - 7 +
				(((pByteEnd - pBytes) >> 1 < pCharEnd - pChars) ?
				(pByteEnd - pBytes) : (pCharEnd - pChars) << 1));

			// Need new UCHAR* so we can check 4 at a time
			UINT64* longBytes = (UINT64*)pBytes;
			UINT64* longChars = (UINT64*)pChars;

			while (longBytes < pLongEnd)
			{
				if (((UINT64)(0x8000800080008000LL) & *longBytes) != 0)
				{
					// See if all 4 of those are bad...
					// Surrogate pairs are "backwards" for little endian machines read in as one
					// Mask off surrogate bits
					UINT64 uTemp = (0xd800d800d800d800LL & *longBytes) ^ 0xd800d800d800d800LL;

					// Check each of the 4 chars.  0 for those 16 bits means it was a surrogate
					if ((uTemp & 0xFFFF000000000000LL) == 0 ||
						(uTemp & 0x0000FFFF00000000LL) == 0 ||
						(uTemp & 0x00000000FFFF0000LL) == 0 ||
						(uTemp & 0x000000000000FFFFLL) == 0)
					{
						// Does it happen to have 4 good surrogates?
#if BIGENDIAN
						if (((0xdc00dc00dc00dc00LL & *longBytes) ^ 0xd800dc00d800dc00LL) != 0)
#else
						if (((0xdc00dc00dc00dc00LL & *longBytes) ^ 0xdc00d800dc00d800LL) != 0)
#endif // BIGENDIAN
						{
							// If it wasn't 4 good surrogates then fall through to slow loop
							break;
						}

						// Use these 4 lucky ordered surrogate pairs.
					}

					// Use these 4 high, but not surrogates
				}

				// We can use these 4 chars.
				*longChars = *longBytes;
				longBytes++;
				longChars++;
			}

			pChars = (UCHAR*)longChars;
			pBytes = (BYTE*)longBytes;

			if (pBytes >= pByteEnd)
				break;
		}

		// Get 1st byte
		if (lastByte < 0)
		{
			lastByte = *pBytes++;
			continue;
		}

		// Get full char
		UCHAR ch;
		if (m_bBigEndian)
		{
			ch = (UCHAR)(lastByte << 8 | *(pBytes++));
		}
		else
		{
			ch = (UCHAR)(*(pBytes++) << 8 | lastByte);
		}
		lastByte = -1;

		// See if the char's valid
		if (ch >= 0xd800 && ch <= 0xdfff)
		{
			// Was it a high surrogate?
			if (ch <= 0xdbff)
			{
				// Its a high surrogate, if we had one then do fallback for previous one
				if (lastChar > 0)
				{
					// Get fallback for previous high surrogate
					// Note we have to reconstruct bytes because some may have been in decoder
					SequenceT<BYTE> bytesUnknown;
					if (m_bBigEndian)
					{
						bytesUnknown.Append((BYTE)(lastChar >> 8));
						bytesUnknown.Append((BYTE)lastChar);
					}
					else
					{
						bytesUnknown.Append((BYTE)lastChar);
						bytesUnknown.Append((BYTE)(lastChar >> 8));
					}

					if (pFallbackBuffer == NULL)
					{
						if (pDecoder == NULL)
							pFallbackBuffer = GetDecoderFallback()->CreateFallbackBuffer();
						else
							pFallbackBuffer = pDecoder->GetFallbackBuffer();

						// Set our internal fallback interesting things.
						pFallbackBuffer->Initialize(pByteStart, pCharEnd);
					}

					if (!pFallbackBuffer->Fallback(bytesUnknown, pBytes, INOUT pChars))
					{
						// couldn't fall back lonely surrogate
						// We either advanced bytes or chars should == charStart and throw below
						BFX_ASSERT1(pBytes >= pByteStart + 2 || pChars == pCharStart, "Expected bytes to have advanced or no output (bad surrogate)");
						pBytes -= 2;                                    // didn't use these 2 bytes
						pFallbackBuffer->ResetAll();
						BFX_ENSURE(pChars != pCharStart);				// Might throw, if no chars output
						break;                                          // couldn't fallback but didn't throw
					}
				}

				// Ignore the previous high surrogate which fell back already,
				// yet remember the current high surrogate for next time.
				lastChar = ch;
				continue;
			}

			// Its a low surrogate
			if (lastChar == 0)
			{
				// Expected a previous high surrogate
				// Get fallback for this low surrogate
				// Note we have to reconstruct bytes because some may have been in decoder
				SequenceT<BYTE> bytesUnknown;
				if (m_bBigEndian)
				{
					bytesUnknown.Append((BYTE)(ch >> 8));
					bytesUnknown.Append((BYTE)ch);
				}
				else
				{
					bytesUnknown.Append((BYTE)ch);
					bytesUnknown.Append((BYTE)(ch >> 8));
				}

				if (pFallbackBuffer == NULL)
				{
					if (pDecoder == NULL)
						pFallbackBuffer = GetDecoderFallback()->CreateFallbackBuffer();
					else
						pFallbackBuffer = pDecoder->GetFallbackBuffer();

					// Set our internal fallback interesting things.
					pFallbackBuffer->Initialize(pByteStart, pCharEnd);
				}

				if (!pFallbackBuffer->Fallback(bytesUnknown, pBytes, INOUT pChars))
				{
					// couldn't fall back lonely surrogate
					// We either advanced bytes or chars should == charStart and throw below
					BFX_ASSERT1(pBytes >= pByteStart + 2 || pChars == pCharStart, "Expected bytes to have advanced or no output (lonely surrogate)");
					pBytes -= 2;                                    // didn't use these 2 bytes
					pFallbackBuffer->ResetAll();
					BFX_ENSURE(pChars != pCharStart);				// Might throw, if no chars output
					break;                                          // couldn't fallback but didn't throw
				}

				// Didn't throw, ignore this one (we already did its fallback)
				continue;
			}

			// Valid surrogate pair, add our lastChar (will need 2 chars)
			if (pChars >= pCharEnd - 1)
			{
				// couldn't find room for this surrogate pair
				// We either advanced bytes or chars should == pCharStart and throw below
				BFX_ASSERT1((pBytes >= pByteStart + 2 || pChars == pCharStart), "Expected bytes to have advanced or no output (surrogate pair)");
				pBytes-=2;                                      // didn't use these 2 bytes

				// Might throw, if no chars output
				BFX_ENSURE1(pChars != pCharStart, "The output char buffer is too small to contain the decoded characters");
				// Leave lastChar
				break;                                          // couldn't fallback but didn't throw
			}

			*pChars++ = lastChar;
			lastChar = (UCHAR)0;
		}
		else if (lastChar > 0)
		{
			// Had a high surrogate, expected a low surrogate, fall back the high surrogate.
			SequenceT<BYTE> bytesUnknown;
			if (m_bBigEndian)
			{
				bytesUnknown.Append((BYTE)(lastChar >> 8));
				bytesUnknown.Append((BYTE)lastChar);
			}
			else
			{
				bytesUnknown.Append((BYTE)lastChar);
				bytesUnknown.Append((BYTE)(lastChar >> 8));
			}

			if (pFallbackBuffer == NULL)
			{
				if (pDecoder == NULL)
					pFallbackBuffer = GetDecoderFallback()->CreateFallbackBuffer();
				else
					pFallbackBuffer = pDecoder->GetFallbackBuffer();

				// Set our internal fallback interesting things.
				pFallbackBuffer->Initialize(pByteStart, pCharEnd);
			}

			if (!pFallbackBuffer->Fallback(bytesUnknown, pBytes, INOUT pChars))
			{
				// couldn't fall back high surrogate, or char that would be next
				// We either advanced bytes or chars should == charStart and throw below
				BFX_ASSERT1(pBytes >= pByteStart + 2 || pChars == pCharStart, "Expected bytes to have advanced or no output (no low surrogate)");
				pBytes -= 2;									// didn't use these 2 bytes
				pFallbackBuffer->ResetAll();
				BFX_ENSURE(pChars == pCharStart);				// Might throw, if no chars output
				break;                                          // couldn't fallback but didn't throw
			}

			// Not left over now, clear previous high surrogate and continue to add current char
			lastChar = (char)0;
		}

		// Valid char, room for it?
		if (pChars >= pCharEnd)
		{
			// 2 bytes couldn't fall back
			// We either advanced bytes or chars should == pCharStart and throw below
			BFX_ASSERT1((pBytes >= pByteStart + 2 || pChars == pCharStart), "Expected bytes to have advanced or no output (normal)");
			pBytes-=2;                                      // didn't use these bytes

			// Might throw, if no chars output
			BFX_ENSURE1(pChars != pCharStart, "The output char buffer is too small to contain the decoded characters");
			break;                                          // couldn't fallback but didn't throw
		}

		// add it
		*pChars++ = ch;
	}

	// Remember our decoder if we must
	if (lastChar > 0)
	{
		if (lastChar > 0)
		{
			// No hanging high surrogates allowed, do fallback and remove count for it
			SequenceT<BYTE> bytesUnknown;
			if (m_bBigEndian)
			{
				bytesUnknown.Append((BYTE)(lastChar >> 8));
				bytesUnknown.Append((BYTE)lastChar);
			}
			else
			{
				bytesUnknown.Append((BYTE)lastChar);
				bytesUnknown.Append((BYTE)(lastChar >> 8));
			}

			if (pFallbackBuffer == NULL)
			{
				if (pDecoder == NULL)
					pFallbackBuffer = GetDecoderFallback()->CreateFallbackBuffer();
				else
					pFallbackBuffer = pDecoder->GetFallbackBuffer();

				// Set our internal fallback interesting things.
				pFallbackBuffer->Initialize(pByteStart, pCharEnd);
			}

			if (!pFallbackBuffer->Fallback(bytesUnknown, pBytes, INOUT pChars))
			{
				// 2 bytes couldn't fall back
				// We either advanced bytes or chars should == charStart and throw below
				BFX_ASSERT1(pBytes >= pByteStart + 2 || pChars == pCharStart, "Expected bytes to have advanced or no output (decoder)");
				pBytes -= 2;									// didn't use these bytes
				if (lastByte >= 0)
					pBytes --;									// had an extra last byte hanging around
				pFallbackBuffer->ResetAll();
				BFX_ENSURE(pChars != pCharStart);				// Might throw, if no chars output
				// We'll remember these in our decoder though
				pBytes += 2;
				if (lastByte >= 0)
					pBytes++;
				goto End;
			}

			// done with this one
			lastChar = (char)0;
		}

		if (lastByte >= 0)
		{
			if (pFallbackBuffer == NULL)
			{
				if (pDecoder == NULL)
					pFallbackBuffer = GetDecoderFallback()->CreateFallbackBuffer();
				else
					pFallbackBuffer = pDecoder->GetFallbackBuffer();

				// Set our internal fallback interesting things.
				pFallbackBuffer->Initialize(pByteStart, pCharEnd);
			}

			// No hanging odd bytes allowed if must flush
			SequenceT<BYTE> bytesUnknown;
			bytesUnknown.Append((BYTE)lastByte);
			if (!pFallbackBuffer->Fallback(bytesUnknown, pBytes, INOUT pChars))
			{
				// odd byte couldn't fall back
				pBytes --;										// didn't use this byte
				pFallbackBuffer->ResetAll();
				BFX_ENSURE(pChars != pCharStart);				// Might throw, if no chars output
				// didn't throw, but we'll remember it in the decoder
				pBytes ++;
				goto End;
			}

			// Didn't fail, clear buffer
			lastByte = -1;
		}
	}

End:
	// Remember our decoder if we must
	if (pDecoder != NULL)
	{
		BFX_ASSERT1((pDecoder->IsFlushRequired() == false) || ((lastChar == (UCHAR)0) && (lastByte == -1)),
			"Expected no left over chars or bytes if flushing");

		// TODO: pDecoder->m_nBytesUsed = (int)(pBytes - pByteStart);
		pUtf16Decoder->m_lastChar = lastChar;
		pUtf16Decoder->m_lastByte = lastByte;
	}

	// Used to do this the old way
	// memcpy((BYTE*)pChars, pBytes, nByteCount);

	// Shouldn't have anything in fallback buffer for GetChars
	// (don't have to check m_throwOnOverflow for count or chars)
	BFX_ASSERT1(pFallbackBuffer == NULL || pFallbackBuffer->GetRemaining() == 0, "Expected empty fallback buffer at end");

	return (int)(pChars - pCharStart);
}

REFDecoder* REFUTF16Encoding::GetDecoder() const
{
	REFDecoder* pDecoder = new UTF16Decoder(this);
	pDecoder->Autorelease();
	return pDecoder;
}

REFUTF16Encoding::UTF16Decoder::UTF16Decoder(const REFEncoding* pEncoding)
	: REFDecoder(pEncoding)
{
}

REFUTF16Encoding::UTF16Decoder::~UTF16Decoder()
{
}

void REFUTF16Encoding::UTF16Decoder::Reset()
{
	m_lastByte = -1;
	m_lastChar = '\0';
	REFDecoder::Reset();
}

}	//	namespace BFX
